#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Create by Albert_Chen
# CopyRight (py) 2016年 陈超. All rights reserved by Chao.Chen.
# Create on 2016-11-10

__author__ = 'Albert'


import sys
reload(sys)
sys.setdefaultencoding('utf8')

import os
import datetime
import argparse
import urlparse
import sys
import pandas as pd
from pandas import DataFrame, Series


def concat_excel(filepath):
    df_list = []
    for root, dirs, files in os.walk(filepath):
        for my_file in files:
            if my_file.startswith(u'国家级'):
                df = pd.read_excel(os.path.join(root, my_file))
                df[u'类别'] = u'国家级'
                df_list.append(df)

            elif my_file.endswith(u'.xls'):
                df = pd.read_excel(os.path.join(root, my_file))
                df[u'类别'] = my_file.split('-')[0].encode('utf8')
                df_list.append(df)
    df_main = pd.concat(df_list)
    return df_main


def concat_excel_windows(filepath):
    df_list = []
    for root, dirs, files in os.walk(filepath):
        for my_file in files:
            if my_file.startswith('._'):
                continue
            elif my_file.startswith(u'国家级'.encode('gbk')):
                df = pd.read_excel(os.path.join(root, my_file))
                df[u'类别'] = u'国家级'
                df_list.append(df)

            elif my_file.endswith('.xls'):
                df = pd.read_excel(os.path.join(root, my_file))
                df[u'类别'] = my_file.split('-')[0].decode('gbk').encode('utf8')
                df_list.append(df)
    df_main = pd.concat(df_list)
    return df_main


def create_pivot(df):

    df['MainUrl'] = df['PageUrl'].map(lambda x:get_domain(x) if isinstance(x,str) else "")
    table = pd.pivot_table(
        df, index=[u'类别', u'网站名称', u'MainUrl'], values= u'PageUrl', aggfunc="count", margins=True
    )
    return table

def save_table(report, filepath):

    if sys.platform == 'win32':
        filename = u"汇  总表新.xlsx".format(datetime.datetime.now().strftime("%Y%m%d")).decode('utf8').encode('gbk')
    else:
        filename = u"汇总表新.xlsx".format(datetime.datetime.now().strftime("%Y%m%d"))
    outfile = os.path.join(filepath, filename)
    report.to_frame(name=u'网站采集数量').to_excel(outfile)


def get_domain(url):
    # try:
    #     segment = urlparse.urlparse(url)
    #     main_url = "%s://%s"%(segment.scheme, segment.netloc)
    # except Exception,e:
    #     main_url = url
    # return main_url

    segment = urlparse.urlparse(url)
    main_url = "%s://%s" % (segment.scheme, segment.netloc)
    return main_url
def main():

    parser = argparse.ArgumentParser(description=u'用来汇总火车头采集下来的数据，并且做成数据交互表')
    parser.add_argument('--input', '-i', dest='input', required=True, help=u'文件路径地址')

    args = parser.parse_args()
    filepath = args.input

    if not os.path.exists(filepath):
        print u"文件夹不存在"
        sys.exit(1)

    if sys.platform == 'win32':
        df_main = concat_excel_windows(filepath)
    else:
        df_main = concat_excel(filepath)
    pivot_table = create_pivot(df_main)
    save_table(pivot_table, filepath)


if __name__ == "__main__":

    main()